package pl.pelcra.cesar.spider.webparser.interfaces;

import java.text.DateFormat;
import java.text.ParseException;
import java.text.SimpleDateFormat;
import java.util.Date;
import java.util.HashMap;

import org.apache.commons.lang.NullArgumentException;
import org.apache.log4j.Logger;
import org.jsoup.Jsoup;
import org.jsoup.helper.StringUtil;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.nodes.Node;
import org.jsoup.nodes.TextNode;
import org.jsoup.select.Elements;
import org.jsoup.select.NodeTraversor;
import org.jsoup.select.NodeVisitor;

import pl.pelcra.cesar.spider.webcrawler.CSWebURL;
import pl.pelcra.cesar.tools.Tools;
import pl.pelcra.cesar.webrecorder.database.tables.DBWeb_Record;

/**
 * Class with base function required to parse page. Class from configuration
 * file should extends this class.
 * 
 * @author margielewski
 * 
 */
public class BasePageParser
{

  private String                  author;
  private String                  authorTag;

  private String                  category;
  private String                  content;

  private Elements                contentElements;
  private String                  contentTag;
  private Date                    crawl_date;
  private String                  doc_id;
  private String                  errorDetails;

  private String                  errorMsg;
  private String                  hash;
  private String                  html;
  private String                  initElementTag;
  private boolean                 isGetPageSuccess = false;
  private String                  language;
  private long                    lastFetchTime    = 0;
  private final Logger            logger           = Logger.getLogger(BasePageParser.class);
  private Element                 mainElement;
  private HashMap<String, String> mapParams        = new HashMap<String, String>();
  private String                  notes;
  private String                  page_date;
  private String                  page_dateTag;
  private String                  page_url;
  private IPageParserHelper       pageParserHelper;
  private Date                    parsed_pagedate;

  private String                  parsed_pagedateFormat;
  private int                     politenessDelay  = 0;
  private String                  site_url;
  private String                  source_id;
  private String                  title;

  private String                  titleTag;
  private String                  urlAttachment;
  private CSWebURL                webURL;

  /**
   * parameterless constructor
   */
  public BasePageParser()
  {

  }

  /**
   * BasePageParser constructor with two parameters required to initilize class.
   * 
   * @param webURL
   *          url link to parsed page
   * @param initElementTag
   *          start html tag that contains all page content
   */
  public BasePageParser(CSWebURL webURL, String initElementTag)
  {
    this.initElementTag = initElementTag;
    this.webURL = webURL;

    this.init();
  }

  /**
   * 
   * @param parameters
   *          String value of parameters from configuration file
   * @param politenessDelay
   *          ask timeout for next page
   */
  public BasePageParser(String parameters, int politenessDelay)
  {
    parseParameters(parameters);
    this.setPolitenessDelay(politenessDelay);
  }

  /**
   * add to DBWeb_Record object string value of remark about parsed page
   * 
   * @param notes
   *          string value of remark
   */
  public void addNotes(String notes)
  {
    this.notes += notes;
  }

  private void clearData()
  {
    this.initElementTag = null;
    this.webURL = null;
    this.mainElement = null;
    this.contentElements = null;
    this.isGetPageSuccess = false;

    this.category = null;
    this.doc_id = null;
    this.language = null;
    this.page_url = null;
    this.site_url = null;
    this.source_id = null;
    this.hash = null;
    this.page_date = null;
    this.parsed_pagedate = null;
    this.crawl_date = null;
    this.title = null;
    this.author = null;
    this.content = null;
    this.html = null;
    this.notes = null;
    this.urlAttachment = null;

    this.errorMsg = null;
    this.errorDetails = null;
  }

  public boolean filterContent(Element element)
  {
    if (this.getPageParserHelper() != null) { return this.getPageParserHelper().filterContent(
        element); }
    return true;
  }

  private String getAuthor()
  {
    if (this.author == null || this.author.isEmpty())
    {
      this.setAuthor();
    }
    return this.author;
  }

  private String getCategory()
  {
    return this.category;
  }

  private String getContent()
  {
    if (this.content == null || this.content.isEmpty())
    {
      this.setContent();
    }
    return this.content;
  }

  /**
   * 
   * @return return object Elements from Content
   */
  public Elements getContentElements()
  {
    if (this.contentElements == null)
    {
      this.setContent();
    }
    return this.contentElements;
  }

  private Date getCrawl_date()
  {
    return this.crawl_date;
  }

  /**
   * function creates DBWeb_Record object from rules provided by class inherited
   * 
   * @return
   */
  public DBWeb_Record getDBWeb_Record()
  {
    DBWeb_Record tmpDBWeb_Record = new DBWeb_Record();

    tmpDBWeb_Record.setDoc_id(this.getDoc_id());
    tmpDBWeb_Record.setCategory(this.getCategory());
    tmpDBWeb_Record.setLanguage(this.getLanguage());
    tmpDBWeb_Record.setPage_url(this.getPage_url());
    tmpDBWeb_Record.setSite_url(this.getSite_url());
    tmpDBWeb_Record.setSource_id(this.getSource_id());
    tmpDBWeb_Record.setHash(this.getHash());
    tmpDBWeb_Record.setPage_date(this.getPage_date());
    tmpDBWeb_Record.setParsed_pagedate(this.getParsed_pagedate());
    tmpDBWeb_Record.setCrawl_date(this.getCrawl_date());
    tmpDBWeb_Record.setTitle(this.getTitle());
    tmpDBWeb_Record.setAuthor(this.getAuthor());
    tmpDBWeb_Record.setContent(this.getContent());
    tmpDBWeb_Record.setHtml(this.getHtml());
    tmpDBWeb_Record.setNotes(this.getNotes());
    tmpDBWeb_Record.setUrlAttachment(this.getUrlAttachment());

    return tmpDBWeb_Record;
  }

  private String getDoc_id()
  {
    return this.doc_id;
  }

  /**
   * looking for tagValue in startElement
   * 
   * @param startElement
   *          Element where looking for the tagValue
   * @param tagValue
   *          html tag
   * @return return object Element with tagValue or return null
   */
  public Element getElement(Element startElement, String tagValue)
  {
    Elements elements = startElement.select(tagValue);
    if (elements != null)
    {
      Element element = elements.first();
      if (element != null) { return element; }
    }
    return null;
  }

  /**
   * looking for tagValue in mainElement
   * 
   * @param tagValue
   *          html tag
   * @return return object Element with tagValue or return null
   */
  public Element getElement(String tagValue)
  {
    Elements elements = this.mainElement.select(tagValue);
    if (elements != null)
    {
      Element element = elements.first();
      if (element != null) { return element; }
    }
    return null;
  }

  private Elements getElements(String tagValue)
  {
    Elements elements = this.mainElement.select(tagValue);
    if (elements != null) { return elements; }
    return null;
  }

  /**
   * 
   * @return return error details of parsing page
   */
  public String getErrorDetails()
  {
    return this.errorDetails;
  }

  /**
   * 
   * @return return error message of parsing page
   */
  public String getErrorMsg()
  {
    return this.errorMsg;
  }

  private String getHash()
  {
    return this.hash;
  }

  private String getHtml()
  {
    return this.html;
  }

  /**
   * 
   * @return return langauge of parsed page
   */
  public String getLanguage()
  {
    return this.language;
  }

  /**
   * 
   * @return return mainElement
   */
  public Element getMainElement()
  {
    return this.mainElement;
  }

  /**
   * 
   * @return return map key - value of parameters from configuration gile
   */
  public HashMap<String, String> getMapParams()
  {
    return this.mapParams;
  }

  private String getNotes()
  {
    return this.notes;
  }

  /**
   * 
   * @return return string value of date from page
   */
  public String getPage_date()
  {
    if (this.page_date == null || this.page_date.isEmpty())
    {
      this.setPage_date();
    }
    return this.page_date;
  }

  /**
   * 
   * @return return url of parsed page
   */
  public String getPage_url()
  {
    return this.page_url;
  }

  private IPageParserHelper getPageParserHelper()
  {
    return this.pageParserHelper;
  }

  private Date getParsed_pagedate()
  {
    if (this.parsed_pagedate == null)
    {
      this.setParsed_pagedate();
    }
    return this.parsed_pagedate;
  }

  /**
   * 
   * @return return date format which is use parse date from page
   */
  public String getParsed_pagedateFormat()
  {
    return this.parsed_pagedateFormat;
  }

  private int getPolitenessDelay()
  {
    return this.politenessDelay;
  }

  private String getSite_url()
  {
    return this.site_url;
  }

  private String getSource_id()
  {
    return this.source_id;
  }

  private String getTitle()
  {
    if (this.title == null || this.title.isEmpty())
    {
      this.setTitle();
    }
    return this.title;
  }

  private String getUrlAttachment()
  {
    return this.urlAttachment;
  }

  private String getValue(Element startElement, String tagValue)
  {
    if (tagValue != null && !tagValue.isEmpty())
    {
      Element element = this.getElement(startElement, tagValue);
      if (element != null) { return element.text(); }
    }
    return "";
  }

  private String getValue(String tagValue)
  {
    if (tagValue != null && !tagValue.isEmpty())
    {
      Element element = this.getElement(tagValue);
      if (element != null) { return element.text(); }
    }
    return "";
  }

  private void init()
  {
    try
    {

      long now = (new Date()).getTime();
      if (now - this.lastFetchTime < this.getPolitenessDelay())
      {
        Tools.threadSleep(this.getPolitenessDelay() - (now - this.lastFetchTime));
      }
      this.lastFetchTime = (new Date()).getTime();

      Document doc = Jsoup.connect(this.webURL.getURL()).get();
      if (doc == null) { throw new NullArgumentException("Document doc is null"); }

      Elements elements = doc.select(this.initElementTag);
      if (elements == null || (elements != null && elements.size() == 0)) { throw new NullArgumentException(
          "Elements elements is null"); }

      this.mainElement = elements.first();
      if (this.mainElement == null) { throw new NullArgumentException(
          "MainElement mainElement is null"); }

      // set default values;
      this.setPage_url();
      this.setSite_url();
      this.setHash();
      this.setCrawl_date();
      this.setHtml();

      this.isGetPageSuccess = true;
    } catch (Exception e)
    {
      this.errorMsg = e.getMessage();
      this.errorDetails = String.format("Error with parsing object '%s' Message='%s'",
          this.webURL.getURL(), this.errorMsg);
      this.isGetPageSuccess = false;
      this.logger.info(this.errorDetails);
    }
  }

  /**
   * initialize all objects in base class
   * 
   * @param webURL
   *          url link to parsed page
   * @param initElementTag
   *          start html tag that contains all page content
   */
  public void init(CSWebURL webURL, String initElementTag)
  {
    this.clearData();

    this.initElementTag = initElementTag;
    this.webURL = webURL;

    this.init();
  }

  /**
   * initialize all object in inherit class. this method should be overwritten
   * otherwise it throws error.
   * 
   * @param webURL
   *          url link to parsed page
   * @param language
   *          page language
   * @param doc_id
   *          document page id
   */
  public void init(CSWebURL webURL, String language, String doc_id)
  {
    throw new UnsupportedOperationException(
        "please overwritte method 'init(WikiWebURL webURL, String language, String doc_id)'");
  }

  /**
   * 
   * @return return true if during parsing page was no errors, false when parser
   *         encounters some error
   */
  public boolean isGetPageSuccess()
  {
    return this.isGetPageSuccess;
  }

  private void parseParameters(String parameters)
  {
    this.mapParams = Tools.parseParameters(parameters);
  }

  private void setAuthor()
  {
    this.author = this.getValue(this.authorTag);
  }

  /**
   * set author html tag
   * 
   * @param authorTag
   *          tag where parser can find author data
   */
  public void setAuthorTag(String authorTag)
  {
    this.authorTag = authorTag;
  }

  /**
   * set category of page
   * 
   * @param category
   *          string value of page category
   */
  public void setCategory(String category)
  {
    this.category = category;
  }

  private void setContent()
  {
    this.contentElements = this.getElements(this.contentTag);
    this.content = new String();
    for (Element element : this.contentElements)
    {
      if (this.filterContent(element))
      {
        content += this.getPlainText(element);
      }
    }
  }

  /**
   * set content html tag
   * 
   * @param contentTag
   *          tag where parser can find content page data
   */
  public void setContentTag(String contentTag)
  {
    this.contentTag = contentTag;
  }

  private void setCrawl_date()
  {
    this.crawl_date = new Date();
  }

  /**
   * set doc id of page
   * 
   * @param docId
   *          string value of page id
   */
  public void setDoc_id(String docId)
  {
    if (docId != null && !docId.isEmpty())
    {
      this.doc_id = docId;
    } else
    {
      this.doc_id = this.webURL.getPath();
    }
  }

  private void setHash()
  {
    this.hash = this.language + this.mainElement.text();
  }

  private void setHtml()
  {
    this.html = this.mainElement.html();
  }

  /**
   * set language of page
   * 
   * @param language
   *          string value of page language
   */
  public void setLanguage(String language)
  {
    this.language = language;
  }

  /**
   * set notes for page
   * 
   * @param notes
   *          string value page notes
   */
  public void setNotes(String notes)
  {
    this.notes = notes;
  }

  private void setPage_date()
  {
    this.page_date = this.getValue(this.page_dateTag);
    this.page_date = Tools.replaceNonBreakingSpace(this.page_date);
  }

  /**
   * set date html tag
   * 
   * @param page_dateTag
   *          tag where parser can find date of page
   */
  public void setPage_dateTag(String page_dateTag)
  {
    this.page_dateTag = page_dateTag;
  }

  private void setPage_url()
  {
    this.page_url = this.webURL.getURL();
  }

  public void setPageParserHelper(IPageParserHelper pageParserHelper)
  {
    this.pageParserHelper = pageParserHelper;
  }

  /**
   * set parsed page date using parsed_pagedateFormat and date of page
   */
  public void setParsed_pagedate()
  {
    try
    {
      DateFormat formatter = new SimpleDateFormat(this.parsed_pagedateFormat);
      this.parsed_pagedate = formatter.parse(this.getPage_date());

    } catch (ParseException e)
    {
      this.logger.info(String.format("Error of parisng date using format %s",
          this.parsed_pagedateFormat));
      this.parsed_pagedate = null;
    }
  }

  /**
   * set value of page date
   * 
   * @param parsed_pagedate
   *          date of page parsed to Date object
   */
  public void setParsed_pagedate(Date parsed_pagedate)
  {
    this.parsed_pagedate = parsed_pagedate;
  }

  /**
   * set date format to parse string value of date
   * 
   * @param parsed_pagedateFormat
   *          value of date format
   */
  public void setParsed_pagedateFormat(String parsed_pagedateFormat)
  {
    this.parsed_pagedateFormat = parsed_pagedateFormat;
    setParsed_pagedate();
  }

  private void setPolitenessDelay(int politenessDelay)
  {
    this.politenessDelay = politenessDelay;
  }

  private void setSite_url()
  {
    this.site_url = this.webURL.getDomain();
  }

  /**
   * set source page id
   * 
   * @param source_id
   *          id of source page
   */
  public void setSource_id(String source_id)
  {
    this.source_id = source_id;
  }

  private void setTitle()
  {
    this.title = this.getValue(this.titleTag);
  }

  /**
   * set title html tag
   * 
   * @param titleTag
   *          tag where parser can find title of page
   */
  public void setTitleTag(String titleTag)
  {
    this.titleTag = titleTag;
  }

  /**
   * set url attachment
   * 
   * @param urlAttachment
   *          url for page in pdf
   */
  public void setUrlAttachment(String urlAttachment)
  {
    this.urlAttachment = urlAttachment;
  }

  protected String getPlainText(Element element)
  {
    PlainTextVisitor formatter = new PlainTextVisitor();
    NodeTraversor traversor = new NodeTraversor(formatter);
    traversor.traverse(element);
    return formatter.toString();
  }

  private class PlainTextVisitor implements NodeVisitor
  {
    private StringBuilder text = new StringBuilder();

    public void head(Node node, int depth)
    {
      if (node instanceof TextNode)
      {
        if (text.length() > 0 && (text.codePointAt(text.length() - 1) != 10))
        {
          text.append(' ');
        }
        append(((TextNode) node).text());
      }
    }

    public void tail(Node node, int depth)
    {
      String name = node.nodeName();
      if (StringUtil.in(name, "br", "td", "th"))
      {
        text.append("\n");
      } else if (StringUtil.in(name, "p", "h1", "h2", "h3", "h4", "h5", "div"))
      {
        text.append("\n\n");
      }
    }

    private void append(String str)
    {
      String s = str.trim();
      if (StringUtil.isBlank(s) || s.equals("&nbsp;")) return;
      text.append(s);
    }

    public String toString()
    {
      return text.toString();
    }
  }
}
